{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-19T16:00:02.255265Z",
     "iopub.status.busy": "2021-07-19T16:00:02.254700Z",
     "iopub.status.idle": "2021-07-19T16:00:03.762870Z",
     "shell.execute_reply": "2021-07-19T16:00:03.762356Z",
     "shell.execute_reply.started": "2021-07-19T16:00:02.255214Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# pandas 数据集读取，dataframe形式的\n",
    "import pandas as pd\n",
    "# 文件读取\n",
    "import codecs\n",
    "\n",
    "train_df = pd.read_csv('/home/lyz/work/kaggle/kaggle-quora-question-pairs/input/train.csv.zip')\n",
    "\n",
    "train_df = train_df[train_df['question2'].apply(lambda x: isinstance(x, str))]\n",
    "train_df = train_df[train_df['question1'].apply(lambda x: isinstance(x, str))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-19T16:00:07.299103Z",
     "iopub.status.busy": "2021-07-19T16:00:07.298517Z",
     "iopub.status.idle": "2021-07-19T16:00:07.315124Z",
     "shell.execute_reply": "2021-07-19T16:00:07.314648Z",
     "shell.execute_reply.started": "2021-07-19T16:00:07.299053Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>What is the story of Kohinoor (Koh-i-Noor) Dia...</td>\n",
       "      <td>What would happen if the Indian government sto...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>How can I increase the speed of my internet co...</td>\n",
       "      <td>How can Internet speed be increased by hacking...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>Why am I mentally very lonely? How can I solve...</td>\n",
       "      <td>Find the remainder when [math]23^{24}[/math] i...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>Which one dissolve in water quikly sugar, salt...</td>\n",
       "      <td>Which fish would survive in salt water?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  qid1  qid2                                          question1  \\\n",
       "0   0     1     2  What is the step by step guide to invest in sh...   \n",
       "1   1     3     4  What is the story of Kohinoor (Koh-i-Noor) Dia...   \n",
       "2   2     5     6  How can I increase the speed of my internet co...   \n",
       "3   3     7     8  Why am I mentally very lonely? How can I solve...   \n",
       "4   4     9    10  Which one dissolve in water quikly sugar, salt...   \n",
       "\n",
       "                                           question2  is_duplicate  \n",
       "0  What is the step by step guide to invest in sh...             0  \n",
       "1  What would happen if the Indian government sto...             0  \n",
       "2  How can Internet speed be increased by hacking...             0  \n",
       "3  Find the remainder when [math]23^{24}[/math] i...             0  \n",
       "4            Which fish would survive in salt water?             0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-11T09:35:08.358976Z",
     "start_time": "2021-03-11T09:35:08.347065Z"
    },
    "execution": {
     "iopub.execute_input": "2021-07-11T03:26:30.596664Z",
     "iopub.status.busy": "2021-07-11T03:26:30.596139Z",
     "iopub.status.idle": "2021-07-11T03:26:30.602839Z",
     "shell.execute_reply": "2021-07-11T03:26:30.601647Z",
     "shell.execute_reply.started": "2021-07-11T03:26:30.596618Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from sklearn.model_selection import train_test_split\n",
    "from torch.utils.data import Dataset, DataLoader, TensorDataset\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-11T09:35:09.263768Z",
     "start_time": "2021-03-11T09:35:09.227357Z"
    },
    "execution": {
     "iopub.execute_input": "2021-07-11T03:26:31.548890Z",
     "iopub.status.busy": "2021-07-11T03:26:31.548377Z",
     "iopub.status.idle": "2021-07-11T03:26:31.583657Z",
     "shell.execute_reply": "2021-07-11T03:26:31.582726Z",
     "shell.execute_reply.started": "2021-07-11T03:26:31.548845Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 划分为训练集和验证集\n",
    "# stratify 按照标签进行采样，训练集和验证部分同分布\n",
    "q1_train, q1_val, q2_train, q2_val, train_label, test_label =  train_test_split(\n",
    "    train_df['question1'].iloc[:5000], \n",
    "    train_df['question2'].iloc[:5000],\n",
    "    train_df['is_duplicate'].iloc[:5000],\n",
    "    test_size=0.2, \n",
    "    stratify=train_df['is_duplicate'].iloc[:5000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-11T03:26:33.964416Z",
     "iopub.status.busy": "2021-07-11T03:26:33.963891Z",
     "iopub.status.idle": "2021-07-11T03:26:33.968563Z",
     "shell.execute_reply": "2021-07-11T03:26:33.967514Z",
     "shell.execute_reply.started": "2021-07-11T03:26:33.964372Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# input_ids：字的编码\n",
    "# token_type_ids：标识是第一个句子还是第二个句子\n",
    "# attention_mask：标识是不是填充"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-11T09:35:19.935035Z",
     "start_time": "2021-03-11T09:35:09.908638Z"
    },
    "execution": {
     "iopub.execute_input": "2021-07-11T03:26:34.430435Z",
     "iopub.status.busy": "2021-07-11T03:26:34.429927Z",
     "iopub.status.idle": "2021-07-11T03:26:43.372779Z",
     "shell.execute_reply": "2021-07-11T03:26:43.372237Z",
     "shell.execute_reply.started": "2021-07-11T03:26:34.430391Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/lyz/.local/lib/python3.6/site-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.26.6) or chardet (3.0.4) doesn't match a supported version!\n",
      "  RequestsDependencyWarning)\n"
     ]
    }
   ],
   "source": [
    "# pip install transformers\n",
    "# transformers bert相关的模型使用和加载\n",
    "from transformers import BertTokenizer\n",
    "# 分词器，词典\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "train_encoding = tokenizer(list(q1_train), list(q2_train), \n",
    "                           truncation=True, padding=True, max_length=100)\n",
    "test_encoding = tokenizer(list(q1_val), list(q2_val), \n",
    "                          truncation=True, padding=True, max_length=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-11T09:35:43.578135Z",
     "start_time": "2021-03-11T09:35:43.571452Z"
    },
    "execution": {
     "iopub.execute_input": "2021-07-11T03:26:45.419560Z",
     "iopub.status.busy": "2021-07-11T03:26:45.418998Z",
     "iopub.status.idle": "2021-07-11T03:26:45.428494Z",
     "shell.execute_reply": "2021-07-11T03:26:45.427831Z",
     "shell.execute_reply.started": "2021-07-11T03:26:45.419512Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 数据集读取\n",
    "class QuoraDataset(Dataset):\n",
    "    def __init__(self, encodings, labels):\n",
    "        self.encodings = encodings\n",
    "        self.labels = labels\n",
    "    \n",
    "    # 读取单个样本\n",
    "    def __getitem__(self, idx):\n",
    "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
    "        item['labels'] = torch.tensor(int(self.labels[idx]))\n",
    "        return item\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "\n",
    "train_dataset = QuoraDataset(train_encoding, list(train_label))\n",
    "test_dataset = QuoraDataset(test_encoding, list(test_label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-11T09:35:44.110121Z",
     "start_time": "2021-03-11T09:35:44.104871Z"
    },
    "execution": {
     "iopub.execute_input": "2021-07-11T03:26:46.745578Z",
     "iopub.status.busy": "2021-07-11T03:26:46.745059Z",
     "iopub.status.idle": "2021-07-11T03:26:46.751895Z",
     "shell.execute_reply": "2021-07-11T03:26:46.750831Z",
     "shell.execute_reply.started": "2021-07-11T03:26:46.745534Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 精度计算\n",
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-11T09:58:49.027161Z",
     "start_time": "2021-03-11T09:58:45.317009Z"
    },
    "execution": {
     "iopub.execute_input": "2021-07-11T03:26:47.539228Z",
     "iopub.status.busy": "2021-07-11T03:26:47.538711Z",
     "iopub.status.idle": "2021-07-11T03:27:22.719607Z",
     "shell.execute_reply": "2021-07-11T03:27:22.718555Z",
     "shell.execute_reply.started": "2021-07-11T03:26:47.539184Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForNextSentencePrediction: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
      "- This IS expected if you are initializing BertForNextSentencePrediction from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForNextSentencePrediction from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertForNextSentencePrediction, AdamW, get_linear_schedule_with_warmup\n",
    "model = BertForNextSentencePrediction.from_pretrained('bert-base-uncased')\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "model.to(device)\n",
    "\n",
    "# 单个读取到批量读取\n",
    "train_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)\n",
    "test_dataloader = DataLoader(test_dataset, batch_size=16, shuffle=True)\n",
    "\n",
    "# 优化方法\n",
    "optim = AdamW(model.parameters(), lr=2e-5)\n",
    "total_steps = len(train_loader) * 1\n",
    "scheduler = get_linear_schedule_with_warmup(optim, \n",
    "                                            num_warmup_steps = 0, # Default value in run_glue.py\n",
    "                                            num_training_steps = total_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-11T09:44:22.077501Z",
     "start_time": "2021-03-11T09:39:16.473609Z"
    },
    "execution": {
     "iopub.execute_input": "2021-07-11T03:27:34.405355Z",
     "iopub.status.busy": "2021-07-11T03:27:34.404795Z",
     "iopub.status.idle": "2021-07-11T03:29:07.358171Z",
     "shell.execute_reply": "2021-07-11T03:29:07.357656Z",
     "shell.execute_reply.started": "2021-07-11T03:27:34.405308Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------Epoch: 0 ----------------\n",
      "epoth: 0, iter_num: 100, loss: 0.4996, 40.00%\n",
      "epoth: 0, iter_num: 200, loss: 0.4844, 80.00%\n",
      "Epoch: 0, Average training loss: 0.5836\n",
      "Accuracy: 0.7460\n",
      "Average testing loss: 0.4886\n",
      "-------------------------------\n",
      "------------Epoch: 1 ----------------\n",
      "epoth: 1, iter_num: 100, loss: 0.5558, 40.00%\n",
      "epoth: 1, iter_num: 200, loss: 0.2994, 80.00%\n",
      "Epoch: 1, Average training loss: 0.4181\n",
      "Accuracy: 0.7460\n",
      "Average testing loss: 0.4895\n",
      "-------------------------------\n"
     ]
    }
   ],
   "source": [
    "# 训练函数\n",
    "def train():\n",
    "    model.train()\n",
    "    total_train_loss = 0\n",
    "    iter_num = 0\n",
    "    total_iter = len(train_loader)\n",
    "    for batch in train_loader:\n",
    "        # 正向传播\n",
    "        optim.zero_grad()\n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        labels = batch['labels'].to(device)\n",
    "        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
    "        loss = outputs[0]\n",
    "        total_train_loss += loss.item()\n",
    "        \n",
    "        # 反向梯度信息\n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "        \n",
    "        # 参数更新\n",
    "        optim.step()\n",
    "        scheduler.step()\n",
    "\n",
    "        iter_num += 1\n",
    "        if(iter_num % 100==0):\n",
    "            print(\"epoth: %d, iter_num: %d, loss: %.4f, %.2f%%\" % (epoch, iter_num, loss.item(), iter_num/total_iter*100))\n",
    "        \n",
    "    print(\"Epoch: %d, Average training loss: %.4f\"%(epoch, total_train_loss/len(train_loader)))\n",
    "    \n",
    "def validation():\n",
    "    model.eval()\n",
    "    total_eval_accuracy = 0\n",
    "    total_eval_loss = 0\n",
    "    for batch in test_dataloader:\n",
    "        with torch.no_grad():\n",
    "            # 正常传播\n",
    "            input_ids = batch['input_ids'].to(device)\n",
    "            attention_mask = batch['attention_mask'].to(device)\n",
    "            labels = batch['labels'].to(device)\n",
    "            outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
    "        \n",
    "        loss = outputs[0]\n",
    "        logits = outputs[1]\n",
    "\n",
    "        total_eval_loss += loss.item()\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        label_ids = labels.to('cpu').numpy()\n",
    "        total_eval_accuracy += flat_accuracy(logits, label_ids)\n",
    "        \n",
    "    avg_val_accuracy = total_eval_accuracy / len(test_dataloader)\n",
    "    print(\"Accuracy: %.4f\" % (avg_val_accuracy))\n",
    "    print(\"Average testing loss: %.4f\"%(total_eval_loss/len(test_dataloader)))\n",
    "    print(\"-------------------------------\")\n",
    "    \n",
    "\n",
    "for epoch in range(2):\n",
    "    print(\"------------Epoch: %d ----------------\" % epoch)\n",
    "    train()\n",
    "    validation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-11T03:44:23.771084Z",
     "iopub.status.busy": "2021-07-11T03:44:23.770509Z",
     "iopub.status.idle": "2021-07-11T03:47:05.838335Z",
     "shell.execute_reply": "2021-07-11T03:47:05.837826Z",
     "shell.execute_reply.started": "2021-07-11T03:44:23.771039Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------Epoch: 0 ----------------\n",
      "epoth: 0, iter_num: 100, loss: 0.3866, 40.00%\n",
      "epoth: 0, iter_num: 200, loss: 0.6430, 80.00%\n",
      "Epoch: 0, Average training loss: 0.4196\n",
      "Accuracy: 0.7450\n",
      "Average testing loss: 0.4897\n",
      "-------------------------------\n",
      "------------Epoch: 1 ----------------\n",
      "epoth: 1, iter_num: 100, loss: 0.4222, 40.00%\n",
      "epoth: 1, iter_num: 200, loss: 0.4139, 80.00%\n",
      "Epoch: 1, Average training loss: 0.4191\n",
      "Accuracy: 0.7460\n",
      "Average testing loss: 0.4873\n",
      "-------------------------------\n"
     ]
    }
   ],
   "source": [
    "class FGM():\n",
    "    def __init__(self, model):\n",
    "        self.model = model\n",
    "        self.backup = {}\n",
    "\n",
    "    def attack(self, epsilon=0.001):\n",
    "        # emb_name这个参数要换成你模型中embedding的参数名\n",
    "        for name, param in self.model.named_parameters():\n",
    "            if param.requires_grad and 'embeddings.word_embeddings' in name:\n",
    "                \n",
    "                # 保存原始参数\n",
    "                self.backup[name] = param.data.clone()\n",
    "                \n",
    "                norm = torch.norm(param.grad)\n",
    "                if norm != 0:\n",
    "                    r_at = epsilon * param.grad / norm\n",
    "                    param.data.add_(r_at)\n",
    "\n",
    "    def restore(self):\n",
    "        # emb_name这个参数要换成你模型中embedding的参数名\n",
    "        for name, param in self.model.named_parameters():\n",
    "            if param.requires_grad and 'embeddings.word_embeddings' in name: \n",
    "                assert name in self.backup\n",
    "                param.data = self.backup[name]\n",
    "        self.backup = {}\n",
    "        \n",
    "# 训练函数\n",
    "def train():\n",
    "    model.train()\n",
    "    total_train_loss = 0\n",
    "    iter_num = 0\n",
    "    total_iter = len(train_loader)\n",
    "    fgm = FGM(model)\n",
    "    for batch in train_loader:\n",
    "        # 正向传播\n",
    "        optim.zero_grad()\n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        labels = batch['labels'].to(device)\n",
    "        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
    "        loss = outputs[0]\n",
    "        total_train_loss += loss.item()\n",
    "        \n",
    "        # 反向梯度信息\n",
    "        loss.backward()\n",
    "                \n",
    "        fgm.attack() # 在embedding上添加对抗扰动\n",
    "        outputs = model(\n",
    "                input_ids=input_ids,\n",
    "                attention_mask=attention_mask,\n",
    "                labels=labels\n",
    "        )\n",
    "        outputs[0].backward() # 反向传播，并在正常的grad基础上，累加对抗训练的梯度\n",
    "        fgm.restore() # 恢复embedding参数\n",
    "        \n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "        \n",
    "        # 参数更新\n",
    "        optim.step()\n",
    "        scheduler.step()\n",
    "\n",
    "        iter_num += 1\n",
    "        if(iter_num % 100==0):\n",
    "            print(\"epoth: %d, iter_num: %d, loss: %.4f, %.2f%%\" % (epoch, iter_num, loss.item(), iter_num/total_iter*100))\n",
    "        \n",
    "    print(\"Epoch: %d, Average training loss: %.4f\"%(epoch, total_train_loss/len(train_loader)))\n",
    "    \n",
    "def validation():\n",
    "    model.eval()\n",
    "    total_eval_accuracy = 0\n",
    "    total_eval_loss = 0\n",
    "    for batch in test_dataloader:\n",
    "        with torch.no_grad():\n",
    "            # 正常传播\n",
    "            input_ids = batch['input_ids'].to(device)\n",
    "            attention_mask = batch['attention_mask'].to(device)\n",
    "            labels = batch['labels'].to(device)\n",
    "            outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
    "        \n",
    "        loss = outputs[0]\n",
    "        logits = outputs[1]\n",
    "\n",
    "        total_eval_loss += loss.item()\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        label_ids = labels.to('cpu').numpy()\n",
    "        total_eval_accuracy += flat_accuracy(logits, label_ids)\n",
    "        \n",
    "    avg_val_accuracy = total_eval_accuracy / len(test_dataloader)\n",
    "    print(\"Accuracy: %.4f\" % (avg_val_accuracy))\n",
    "    print(\"Average testing loss: %.4f\"%(total_eval_loss/len(test_dataloader)))\n",
    "    print(\"-------------------------------\")\n",
    "    \n",
    "\n",
    "for epoch in range(2):\n",
    "    print(\"------------Epoch: %d ----------------\" % epoch)\n",
    "    train()\n",
    "    validation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
